#include <iostream>
#include <chrono>
#include <fstream>
#include <string>
#include <vector>
#include <regex>

#include <dirent.h>
#include <string.h>
#include <iconv.h> 

#include "pool.hpp"

struct Config {
    std::string src_dir;
    std::string dst_dir;
    std::string save_dir;
    std::string tmp_dir;
    std::string py_path;
};

struct Info {
    std::string docno;
    std::string url;
    std::string title;
    std::string content;
    void reset() {
        docno.clear();
        url.clear();
        title.clear();
        content.clear();
    }
};

std::string getTime() {
    time_t timep;
    time (&timep);
    char tmp[64];
    strftime(tmp, sizeof(tmp), "%Y-%m-%d %H:%M:%S",localtime(&timep) );
    return tmp;
}

template< typename ... Args  >
static auto print( Args ... x ) {
    auto printer = []( auto y ) { std::cout << y; };

    static std::mutex mutex;
    std::lock_guard<std::mutex> lock(mutex);

    auto ans =  { 0, ( printer(x), 0 )..., 0 };
    std::cout << std::endl;
    return ans;
}

template< typename ... Args  >
static auto log( Args ... x ) {
    std::ofstream fout( "log.txt", std::ios::app );
    if (!fout) return;
    auto printer = [&fout]( auto y ) { fout << y; };

    auto time_str = getTime();

    static std::mutex mutex;
    std::lock_guard<std::mutex> lock(mutex);

    fout << "[" << time_str << "] ";
    auto ans =  { 0, ( printer(x), 0 )..., 0 };
    fout << std::endl;
    fout.close();
}

class Tools {
public:
    static void test(std::shared_ptr<Config> cfg) {
        print("DIR SRC : ", cfg->src_dir);
        print("DIR DST : ", cfg->dst_dir);
        print("DIR SAVE: ", cfg->save_dir);
        print("DIR TEMP: ", cfg->tmp_dir);
        print("PATH PY : ", cfg->py_path);
    }

    //获取目录下的文件及子目录, dirpath不要有最后/
    static bool list_dir_file(
            const std::string& dirpath,
            std::vector<std::string> *files,
            std::vector<std::string> *dirs,
            bool recurse = true, // 递归
            bool excludedotdir = true) { //排除.开始的目录
        DIR* task = opendir(dirpath.c_str());
        if (!task || (!files && !dirs)) {
            perror(dirpath.c_str());
            return false;
        }

        struct dirent* dent;
        while ((dent = readdir(task))) {
            if (strcmp(dent->d_name, ".") == 0 ||
                strcmp(dent->d_name, "..") == 0)
                continue;
            if (excludedotdir && dent->d_type & DT_DIR && dent->d_name[0] == '.')
                continue;
            if (dent->d_type & DT_REG) {
                if (files) {
                    files->push_back(dirpath);
                    files->back() += '/';
                    files->back() += dent->d_name;
                }
            }
            if (dent->d_type & DT_DIR) {
                if (dirs) {
                    dirs->push_back(dirpath);
                    dirs->back() += '/';
                    dirs->back() += dent->d_name;
                }
                if (recurse) {
                    std::string path2(dirpath);
                    path2 += '/';
                    path2 += dent->d_name;
                    list_dir_file(path2, files, dirs, recurse, excludedotdir);
                }
            }
        }
        closedir(task);
        return true;
    }

    static bool is_file(const std::string &f, const std::string &ch) {
        if (ch.empty()) return true;
        int l = f.length();
        int lch = ch.length();
        for (int i=lch-1; i>=0; --i) {
            if (f[l-i-1] != ch[lch-i-1])
                return false;
        }
        return true;
    }

    static std::shared_ptr<std::vector<std::string>> list_file(
            const std::string &dir,
            const std::string &ch,
            bool sub = true ) {
        auto files = std::make_shared<std::vector<std::string>>();
        std::vector<std::string> tmp;
        Tools::list_dir_file(dir, &tmp, nullptr);
        for (auto&& f : tmp) {
            if (is_file(f, ch))
               files->push_back(f);
        }
        return files;
    }

    static int run_cmd(const std::string &cmd) {
        return system(cmd.c_str());
    }

    static bool u7z(const std::string &file, const std::string &dst) {
        std::string cmd = "7z x -aoa -o";
        cmd += dst;
        cmd += " " + file;
        return run_cmd(cmd) == 0;
    }

    static bool copy(const std::string &src, const std::string &dst) {
        std::string cmd = "cp " + src + " " + dst;
        return run_cmd(cmd)==0;
    }

    static void empty_dir(const std::string &dir) {
        std::string cmd = "rm -rf " + path_join(dir, "*");
        run_cmd(cmd);
    }

    static void rm(const std::string &dir) {
        std::string cmd = "rm -rf " + dir;
        run_cmd(cmd);
    }

    static bool write_file(const std::string &file, const std::string &content, std::ios_base::openmode mode=std::ios::trunc) {
        std::ofstream fout( file, mode );
        if (!fout) return false;
        fout << content;
        fout.close();
        return true;
    }

    static void del_file(const std::string &file) {
        std::remove(file.c_str());
    }

    static std::string path_join(const std::string &root, const std::string &path) {
        std::string full = root;
        if (root[root.length()-1] != '/') {
            full = root + "/";
        }
        if (path[0] == '/') {
            full += path.substr(1);
        } else {
            full += path;
        }
        return std::move(full);
    }

    static std::string path_file_full_name(const std::string &path) {
        auto pos = path.rfind('/');
        if (pos == std::string::npos) {
            return path;
        }
        return std::move(path.substr(pos+1));
    }

    static std::string path_file_name(const std::string &path) {
        auto full = path_file_full_name(path);
        auto pos = full.rfind('.');
        if (pos == std::string::npos) {
            return full;
        }
        return std::move(full.substr(0, pos));
    }

    static std::string path_file_ext(const std::string &path) {
        auto pos = path.rfind('.');
        if (pos == std::string::npos) {
            return std::string();
        }
        return std::move(path.substr(pos+1));
    }

    static int gbk2utf8(const std::string &src, std::string &dst) {
        char *inbuf = (char*)src.c_str();
        size_t inlen = src.length();
        size_t srclen = inlen;
        dst.resize(srclen*3/2+4);
        char* outbuf = (char*)dst.c_str();
        char *srcbuf = inbuf;
        size_t outlen = srclen * 3 / 2 + 4;

        iconv_t cd;
        char **pin = &inbuf;
        char **pout = &outbuf;

        //utf8转换是没有BOM的
        cd = iconv_open("utf8//IGNORE", "gbk");
        if (cd == 0) {
            //perror("iconv_open");
            return -1;
        }

        bool failed = false;
        if (iconv(cd, pin, &inlen, pout, &outlen) == -1) {
            //perror("iconv");
            failed = true;
        }
        iconv_close(cd);
        if (failed) {
            return -1;
        }
        else {
            dst.resize(srclen * 3 / 2 + 4- outlen);
            return 0;
        }
    }

    static bool same_begin(const std::string & long_str, const std::string &short_str) {
        if (short_str.empty() || long_str.empty()) return false;
        int l = long_str.length();
        int lch = short_str.length();
        for (int i=0; i<lch; ++i) {
            if (long_str[i] != short_str[i])
                return false;
        }
        return true;
    }

    // get_html_text("<title>abcdef</title>", "title") => abcdef
    static std::string get_html_text(const std::string &html, const std::string &mark) {
        auto bp = html.find("<"+mark+">");
        if (bp == std::string::npos) return std::string();
        auto ep = html.rfind("</"+mark+">");
        if (ep == std::string::npos) return std::string();
        return html.substr(bp+mark.length()+2, ep-bp-mark.length()-2);
    }

    /*
        a、先将html文本中的所有空格、换行符去掉（因为html中的空格和换行是被忽略的）
        b、将<head>标记中的所有内容去掉
        c、将<script>标记中的所有内容去掉
        d、将<style>标记中的所有内容去掉
        e、将td换成空格，tr,li,br,p 等标记换成换行符
        f、去掉所有以“<>”符号为头尾的标记去掉。
        g、转换&amp;，&nbps;等转义字符换成相应的符号
        h、去掉多余的空格和空行
    */
    static std::string html2text(const std::string &html) {
        static std::mutex mutex;
        std::lock_guard<std::mutex> lock(mutex);
    
        // remove line breaks,tabs
        static std::regex reg1("[\\r\\n\\t]");
        std::string resp = std::regex_replace(html, reg1, "");

        // remove the header
        static std::regex reg2("(<head>).*(</head>)");
        resp = std::regex_replace(resp, reg2, "");

        // remove all scripts
        static std::regex reg3("<script[^>]*?>.*?</script>");
        resp = std::regex_replace(resp, reg3, "");

        // remove all styles
        static std::regex reg4("<style[^>]*?>.*?</style>");
        resp = std::regex_replace(resp, reg4, "");

        //
        static std::regex reg5("<( )*td([^>])*>");
        resp = std::regex_replace(resp, reg5, " ");

        static std::regex reg6("<( )*br( )*>");
        resp = std::regex_replace(resp, reg6, "\r");
        static std::regex reg7("<( )*li( )*>");
        resp = std::regex_replace(resp, reg7, "\r");
        static std::regex reg8("<( )*tr([^>])*>");
        resp = std::regex_replace(resp, reg8, "\r\r");
        static std::regex reg9("<( )*p([^>])*>");
        resp = std::regex_replace(resp, reg9, "\r\r");

        static std::regex reg10("<[^>]*>");
        resp = std::regex_replace(resp, reg10, "");

        static std::regex reg11("&amp;");
        resp = std::regex_replace(resp, reg11, "&");
        static std::regex reg12("&nbsp");
        resp = std::regex_replace(resp, reg12, " ");
        static std::regex reg13("&lt;");
        resp = std::regex_replace(resp, reg13, "<");
        static std::regex reg14("&gt;");
        resp = std::regex_replace(resp, reg14, ">");
        static std::regex reg15("&(.{2,6});");
        resp = std::regex_replace(resp, reg15, "");

        static std::regex reg16("( )+");
        resp = std::regex_replace(resp, reg16, "");
        static std::regex reg17("(\r)( )+(\r)");
        resp = std::regex_replace(resp, reg17, "\r\r");
        static std::regex reg18("(\r\r)+");
        resp = std::regex_replace(resp, reg18, "\r\n");

        return std::move(resp);
    }

    static int64_t getTimeStamp() {
        std::chrono::time_point<std::chrono::system_clock, std::chrono::milliseconds> tp = std::chrono::time_point_cast<std::chrono::milliseconds>(std::chrono::system_clock::now());
        return tp.time_since_epoch().count();
    }
};

class Handle {
private:
    static std::shared_ptr<Config> m_cfg;
public:
    static auto init() {
        m_cfg->src_dir = "/home/dzlua/tg-data/data/sougouT";
        m_cfg->dst_dir = "/home/dzlua/tg-data/dst";
        m_cfg->save_dir = "/home/dzlua/tg-data/data/tgtxt";
        m_cfg->tmp_dir = "/home/dzlua/tg-data/tmp";
        m_cfg->py_path = "/home/dzlua/tg-data/src/tg-trans-sougou-data/trans/py/handleinfo.py";
        return m_cfg;
    }

    static void start(const std::string &filename, int size, int i) {
        print("正在处理第 ", i, "/", size, " 个： ", filename);

        auto file_full_name = Tools::path_file_full_name(filename);
        auto file_name = Tools::path_file_name(filename);
        auto file_ext = Tools::path_file_ext(filename);
        auto dst_path = Tools::path_join(m_cfg->dst_dir, file_name);
        auto file = Tools::path_join(m_cfg->dst_dir, file_full_name);

        // copy 7z to dst
        /*if (!Tools::copy(filename, file)) {
            print("错误： 复制1失败！第 ", i, "/", size, " 个： ", filename);
            log("错误： 复制1失败！第 ", i, "/", size, " 个： ", filename);
            goto error;
        }*/

        // un7z
        /*if (!Tools::u7z(file, dst_path)) {
            print("错误： 解压失败！第 ", i, "/", size, " 个： ", filename);
            log("错误： 解压失败！第 ", i, "/", size, " 个： ", filename);
            goto error;
        }*/

        {
            // list files
            auto files = Tools::list_file(dst_path, "");
            int j = 1;
            for (auto f=files->begin(); f!=files->end(); ++f, ++j) {
                if (!handle(*f, i, j)) {
                    print("错误： 处理失败！第 ",
                        i, "/", size, ",",
                        j, "/", files->size(),
                        " 个： ", filename);
                    log("错误： 处理失败！第 ",
                        i, "/", size, ",",
                        j, "/", files->size(),
                        " 个： ", filename);
                }
            }
        }

        {
            // copy dst_dir/tgtxt to save_dir
            auto files = Tools::list_file(Tools::path_join(m_cfg->dst_dir, "/tgtxt"), ".tgtxt");
            int k = 1;
            for (auto f=files->begin(); f!=files->end(); ++f, ++k) {
                auto file_full_name = Tools::path_file_full_name(*f);
                if (!Tools::copy(*f, Tools::path_join(m_cfg->save_dir, file_full_name))) {
                    print("错误： 复制2失败！第 ", k, "/", files->size(), " 个： ", *f);
                    log("错误： 复制2失败！第 ", k, "/", files->size(), " 个： ", *f);
                }
            }
        }
        
        print("完成第 ", i, "/", size, " 个： ", filename);
    error:

        // clear
        Tools::rm(filename);
        Tools::rm(dst_path);
        Tools::rm(Tools::path_join(m_cfg->dst_dir, "/tgtxt"));

    }

    static bool handle(const std::string &file, int i, int j) {
        std::ifstream fin(file);
        if (!fin.is_open()) {
            print("错误： 打开问价失败！ ", file);
            log("错误： 打开问价失败！ ", file);
            return false;
        }
        
        std::string line;
        Info info;
        int64_t size_read = 0;
        log("begin:", Tools::getTimeStamp());
        while (!fin.eof()) {
            std::getline(fin, line);
            size_read += line.length();
            if (Tools::same_begin(line, "<doc>")) {
                info.reset();
            } else if (Tools::same_begin(line, "</doc>")) {
                handle_info(info);
                log("end:", Tools::getTimeStamp(), ",size:", size_read);
            } else if (Tools::same_begin(line, "<docno>")) {
                info.docno = Tools::get_html_text(line, "docno");
            } else if (Tools::same_begin(line, "<url>")) {
                info.url = Tools::get_html_text(line, "url");
            } else if (Tools::same_begin(line, "<title>")) {
                /*if (0 != Tools::gbk2utf8(
                        Tools::get_html_text(line, "title"),
                        info.title )) {
                    //info.title = std::move(tmp);
                }*/
                info.title = Tools::get_html_text(line, "title");
            } else {
                info.content.append(line);
            }
        }

        fin.close();
        return true;
    }

    static bool handle_info(Info &info) {
        std::string file = Tools::path_join(m_cfg->tmp_dir, info.docno+".html");
        std::string content = info.url + "\n" + info.title + "\n" + info.docno + "\n";

        if (!Tools::write_file(file, content + info.content)) {
            print("错误： 写入临时文件错误.", info.docno);
            log("错误： 写入临时文件错误.", info.docno);
            return false;
        }

        std::string cmd = "python " + m_cfg->py_path;
        cmd += " " + file;
        cmd += " " + Tools::path_join(m_cfg->dst_dir, "tgtxt");

        /*
            this py script only 1 process.
        */
        static std::mutex mutex;
        std::lock_guard<std::mutex> lock(mutex);

        auto code = Tools::run_cmd(cmd);

        if (code == 0) {
            Tools::del_file(file);
            return true;
        }
        return false;
    }
};

std::shared_ptr<Config> Handle::m_cfg = std::make_shared<Config>();

int main(int argc, char** argv)
{
    // config
    auto cfg = Handle::init();

    // init pool
    dzlua::thread::Pool pool(1);
    std::vector< std::future<int> > results;

    Tools::test(cfg);
    auto files = Tools::list_file(cfg->src_dir, ".7z");
    int i = 1;
    for (auto f=files->begin(); f!=files->end(); ++f, ++i) {
        pool.PostTask([f, files, i] {
            Handle::start(*f, files->size(), i);
        });
    }

    return 0;
}
